{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn import init\n",
    "import math\n",
    "import os\n",
    "import numpy as np\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChannelAttention(nn.Module):\n",
    "    def __init__(self, in_planes, ratio=16):\n",
    "        super(ChannelAttention, self).__init__()\n",
    "        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n",
    "        self.max_pool = nn.AdaptiveMaxPool2d(1)\n",
    "        self.fc1   = nn.Conv2d(in_planes, in_planes // 16, 1, bias=False)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.fc2   = nn.Conv2d(in_planes // 16, in_planes, 1, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x))))\n",
    "        max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x))))\n",
    "        out = avg_out + max_out\t\t# 这里并没有使用到论文中的shared MLP, 而是直接相加了\n",
    "        return self.sigmoid(out)\n",
    "\n",
    "\n",
    "class SpatialAttention(nn.Module):\n",
    "    def __init__(self, kernel_size=7):\n",
    "        super(SpatialAttention, self).__init__()\n",
    "        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'\n",
    "        padding = 3 if kernel_size == 7 else 1\n",
    "        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = torch.mean(x, dim=1, keepdim=True)\n",
    "        max_out, _ = torch.max(x, dim=1, keepdim=True)\n",
    "        x = torch.cat([avg_out, max_out], dim=1)\n",
    "        x = self.conv1(x)\n",
    "        return self.sigmoid(x)\n",
    "def conv3x3(in_planes, out_planes, stride=1):\n",
    "    \"3x3 convolution with padding\"\n",
    "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
    "                     padding=1, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SEModule(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, inplanes, stride=1, downsample=None):\n",
    "        super(SEModule, self).__init__()\n",
    "        self.conv1 = conv3x3(inplanes, inplanes, stride)\n",
    "        self.bn1 = nn.BatchNorm2d(inplanes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = conv3x3(inplanes, inplanes)\n",
    "        self.bn2 = nn.BatchNorm2d(inplanes)\n",
    "\n",
    "        self.ca = ChannelAttention(inplanes)\n",
    "        self.sa = SpatialAttention()\n",
    "\n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        out = self.ca(out) * out\n",
    "       \n",
    "#         out += residual\n",
    "      \n",
    "#         print(out.size())\n",
    "        out = self.sa(out) * out\n",
    "#         print(out.size())\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "\n",
    "        out += residual\n",
    "#         print(out.size())\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv31(ip,op,stride=1):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "def conv32(ip,op,stride=2):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "class Upsample(nn.Module):\n",
    "    def __init__(self,ip,op):\n",
    "        super(Upsample,self).__init__()\n",
    "        self.upsample=nn.Sequential(\n",
    "            conv31(ip,op,1),\n",
    "            nn.Upsample(scale_factor=2,mode='nearest')\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x=self.upsample(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_3x3_bn(inp, oup, stride):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),\n",
    "        nn.BatchNorm2d(oup),\n",
    "        h_swish()\n",
    "    )\n",
    "\n",
    "\n",
    "def conv_1x1_bn(inp, oup):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(inp, oup, 1, 1, 0, bias=False),\n",
    "        nn.BatchNorm2d(oup),\n",
    "        h_swish()\n",
    "    )\n",
    "def conv2d(filter_in, filter_out, kernel_size, groups=1, stride=1):\n",
    "    pad = (kernel_size - 1) // 2 if kernel_size else 0\n",
    "    return nn.Sequential(OrderedDict([\n",
    "        (\"conv\", nn.Conv2d(filter_in, filter_out, kernel_size=kernel_size, stride=stride, padding=pad, groups=groups, bias=False)),\n",
    "        (\"bn\", nn.BatchNorm2d(filter_out)),\n",
    "        (\"relu\", nn.ReLU6(inplace=True)),\n",
    "    ]))\n",
    "\n",
    "def conv_dw(filter_in, filter_out, stride = 1):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(filter_in, filter_in, 3, stride, 1, groups=filter_in, bias=False),\n",
    "        nn.BatchNorm2d(filter_in),\n",
    "        nn.ReLU6(inplace=True),\n",
    "\n",
    "        nn.Conv2d(filter_in, filter_out, 1, 1, 0, bias=False),\n",
    "        nn.BatchNorm2d(filter_out),\n",
    "        nn.ReLU6(inplace=True),\n",
    "    )\n",
    "#---------------------------------------------------#\n",
    "#   Èý´Î¾í»ý¿é\n",
    "#---------------------------------------------------#\n",
    "def make_three_conv(filters_list, in_filters):\n",
    "    m = nn.Sequential(\n",
    "        conv2d(in_filters, filters_list[0], 1),\n",
    "        conv_dw(filters_list[0], filters_list[1]),\n",
    "        conv2d(filters_list[1], filters_list[0], 1),\n",
    "    )\n",
    "    return m\n",
    "\n",
    "#---------------------------------------------------#\n",
    "#   Îå´Î¾í»ý¿é\n",
    "#---------------------------------------------------#\n",
    "def make_five_conv(filters_list, in_filters):\n",
    "    m = nn.Sequential(\n",
    "        conv2d(in_filters, filters_list[0], 1),\n",
    "        conv_dw(filters_list[0], filters_list[1]),\n",
    "        conv2d(filters_list[1], filters_list[0], 1),\n",
    "        conv_dw(filters_list[0], filters_list[1]),\n",
    "        conv2d(filters_list[1], filters_list[0], 1),\n",
    "    )\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Hswish(x,inplace=True):\n",
    "    return x * F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "def Hsigmoid(x,inplace=True):\n",
    "    return F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "\n",
    "class Bottleneck(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,kernel_size,exp_channels,stride,se='True',nl='HS'):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        padding = (kernel_size - 1) // 2\n",
    "        if nl == 'RE':\n",
    "            self.nlin_layer = F.relu6\n",
    "        elif nl == 'HS':\n",
    "            self.nlin_layer = Hswish\n",
    "        self.stride=stride\n",
    "        if se:\n",
    "            self.se=SEModule(exp_channels)\n",
    "        else:\n",
    "            self.se=None\n",
    "        self.conv1=nn.Conv2d(in_channels,exp_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(exp_channels)\n",
    "        self.conv2=nn.Conv2d(exp_channels,exp_channels,kernel_size=kernel_size,stride=stride,\n",
    "                             padding=padding,groups=exp_channels,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(exp_channels)\n",
    "        self.conv3=nn.Conv2d(exp_channels,out_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn3=nn.BatchNorm2d(out_channels)\n",
    "        # ÏÈ³õÊ¼»¯Ò»¸ö¿ÕÐòÁÐ£¬Ö®ºó¸ÄÔìÆä³ÉÎª²Ð²îÁ´½Ó\n",
    "        self.shortcut = nn.Sequential()\n",
    "        # Ö»ÓÐ²½³¤Îª1ÇÒÊäÈëÊä³öÍ¨µÀ²»ÏàÍ¬Ê±²Å²ÉÓÃÌøÔ¾Á¬½Ó(ÏëÒ»ÏÂÌøÔ¾Á´½ÓµÄ¹ý³Ì£¬ÊäÈëÊä³öÍ¨µÀÏàÍ¬Õâ¸öÌøÔ¾Á¬½Ó¾ÍÃ»ÒâÒåÁË)\n",
    "        if stride == 1 and in_channels != out_channels:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                # ÏÂÃæµÄ²Ù×÷¾í»ý²»¸Ä±ä³ß´çºÍÍ¨µÀÊý\n",
    "                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "\n",
    "    def forward(self,x):\n",
    "        out=self.nlin_layer(self.bn1(self.conv1(x)))\n",
    "        if self.se is not None:\n",
    "            out=self.bn2(self.conv2(out))\n",
    "            out=self.nlin_layer(self.se(out))\n",
    "        else:\n",
    "            out = self.nlin_layer(self.bn2(self.conv2(out)))\n",
    "        out = self.bn3(self.conv3(out))\n",
    "        out = out + self.shortcut(x) if self.stride == 1 else out\n",
    "        return out\n",
    "\n",
    "\n",
    "class MobileNetV3_large(nn.Module):\n",
    "    # (out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "    cfg=[\n",
    "        (16,3,16,1,False,'RE'),\n",
    "        (24,3,64,2,False,'RE'),\n",
    "        (24,3,72,1,False,'RE'),\n",
    "        (40,5,72,2,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (80,3,240,2,False,'HS'),\n",
    "        (80,3,200,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (112,3,480,1,True,'HS'),\n",
    "        (112,3,672,1,True,'HS'),\n",
    "        (160,5,672,2,True,'HS'),\n",
    "        (160,5,960,1,True,'HS'),\n",
    "        (160,5,960,1,True,'HS')\n",
    "    ]\n",
    "    def __init__(self,num_classes=5):\n",
    "        super(MobileNetV3_large,self).__init__()\n",
    "        self.conv1=nn.Conv2d(3,16,3,2,padding=1,bias=False)\n",
    "        self.bn1=nn.BatchNorm2d(16)\n",
    "        # ¸ù¾ÝcfgÊý×é×Ô¶¯Éú³ÉËùÓÐµÄBottleneck²ã\n",
    "        self.layers = self._make_layers(in_channels=16)\n",
    "        self.conv2=nn.Conv2d(160,960,1,stride=1,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(960)\n",
    "        # ¾í»ýºó²»¸úBN£¬¾ÍÓ¦¸Ã°ÑbiasÉèÖÃÎªTrue\n",
    "        self.conv3=nn.Conv2d(960,1280,1,1,padding=0,bias=True)\n",
    "        self.conv4=nn.Conv2d(1280,num_classes,1,stride=1,padding=0,bias=True)\n",
    "        \n",
    "        self.down_sample1 = conv_dw(40,112,stride=2)\n",
    "        self.mk1=make_five_conv([112,224],224)\n",
    "        \n",
    "        self.down_sample2 = conv_dw(112,160,stride=2)\n",
    "        self.mk2=make_five_conv([160,320],320)\n",
    "        \n",
    "#         self.w=nn.Parameter(torch.FloatTensor(1),requires_grad=True)\n",
    "#         self.w=torch.FloatTensor(1)\n",
    "        self.weights_try=1\n",
    "        self.weights2_try=1\n",
    "        self.weights3_try=1\n",
    "        \n",
    "#         self.w.data.fill_(1)\n",
    "#         self.weights2_try.data.fill_(1)\n",
    "#         self.weights3_try.data.fill_(1)\n",
    "        \n",
    "    def _make_layers(self,in_channels):\n",
    "        layers=[]\n",
    "        for out_channels,kernel_size,exp_channels,stride,se,nl in self.cfg:\n",
    "            layers.append(\n",
    "                Bottleneck(in_channels,out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "            )\n",
    "            in_channels=out_channels\n",
    "        return nn.Sequential(*layers)\n",
    "                \n",
    "\n",
    "    def forward(self,x):\n",
    "        out=Hswish(self.bn1(self.conv1(x)))\n",
    "        ##############################################\n",
    "        out1=self.layers[:6](out) # 28*28*40\n",
    "        out2=self.layers[6:12](out1) # 14*14*112\n",
    "        out3=self.layers[12:16](out2) # 7*7*160\n",
    "        \n",
    "        out1_dw=self.down_sample1(out1)\n",
    "        out2=torch.cat([out1_dw,out2],1)\n",
    "        out2=self.mk1(out2)\n",
    "        out2_dw=self.down_sample2(out2)\n",
    "        out3=torch.cat([out2_dw,out3],1)\n",
    "        out3=self.mk2(out3)\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        out=self.layers(out)\n",
    "        out=out+out3\n",
    "        out=Hswish(self.bn2(self.conv2(out)))\n",
    "        out=F.avg_pool2d(out,7)\n",
    "        out=Hswish(self.conv3(out))\n",
    "        out=self.conv4(out)\n",
    "        \n",
    "        a,b=out.size(0),out.size(1)\n",
    "        out=out.view(a,b)\n",
    "#         print(self.weights_try)\n",
    "#         return out,out1,out2,out3\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=MobileNetV3_large()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x=torch.randn(2,3,224,224)\n",
    "# y=model(x)\n",
    "# print(y.size())\n",
    "# print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "random.seed(1)\n",
    "\n",
    "class flowerDataset(Dataset):\n",
    "    # 自定义Dataset类，必须继承Dataset并重写__init__和__getitem__函数\n",
    "    def __init__(self, data_dir, transform=None):\n",
    "        \"\"\"\n",
    "            花朵分类任务的Dataset\n",
    "            :param data_dir: str, 数据集所在路径\n",
    "            :param transform: torch.transform，数据预处理，默认不进行预处理\n",
    "        \"\"\"\n",
    "        # data_info存储所有图片路径和标签（元组的列表），在DataLoader中通过index读取样本\n",
    "        self.data_info = self.get_img_info(data_dir)\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        path_img, label = self.data_info[index]\n",
    "        # 打开图片，默认为PIL，需要转成RGB\n",
    "        img = Image.open(path_img).convert('RGB')\n",
    "        # 如果预处理的条件不为空，应该进行预处理操作\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        return img, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data_info)\n",
    "\n",
    "    # 自定义方法，用于返回所有图片的路径以及标签\n",
    "    @staticmethod\n",
    "    def get_img_info(data_dir):\n",
    "        data_info = list()\n",
    "        for root, dirs, _ in os.walk(data_dir):\n",
    "            # 遍历类别\n",
    "            for sub_dir in dirs:\n",
    "                # listdir为列出文件夹下所有文件和文件夹名\n",
    "                img_names = os.listdir(os.path.join(root, sub_dir))\n",
    "                # 过滤出所有后缀名为jpg的文件名（那当然也就把文件夹过滤掉了）\n",
    "#                 img_names = list(filter(lambda x: x.endswith('.png'), img_names))\n",
    "\n",
    "                # 遍历图片\n",
    "                for i in range(len(img_names)):\n",
    "                    img_name = img_names[i]\n",
    "                    path_img = os.path.join(root, sub_dir, img_name)\n",
    "                    # 在该任务中，文件夹名等于标签名\n",
    "                    label = sub_dir\n",
    "                    data_info.append((path_img, int(label)))\n",
    "        return data_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#宏定义一些数据，如epoch数，batchsize等\n",
    "MAX_EPOCH=45\n",
    "BATCH_SIZE=64\n",
    "# LR=0.0001\n",
    "LR=0.001\n",
    "log_interval=30\n",
    "val_interval=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 1/5 数据 ============================\n",
    "# split_dir=os.path.join(\"MobileNetV3-master\",\".\",\"data\",\"splitData\")\n",
    "split_dir=r'/home/fanrz/Desktop/torchmob/'\n",
    "train_dir=r'/home/fanrz/Desktop/testlarge/aptos/tra2/'\n",
    "valid_dir=os.path.join(split_dir,\"val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对训练集所需要做的预处理\n",
    "train_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对验证集所需要做的预处理\n",
    "valid_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建MyDataset实例\n",
    "train_data=flowerDataset(data_dir=train_dir,transform=train_transform)\n",
    "valid_data=flowerDataset(data_dir=valid_dir,transform=valid_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5035\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data.data_info))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建DataLoader\n",
    "# 训练集数据最好打乱\n",
    "# DataLoader的实质就是把数据集加上一个索引号，再返回\n",
    "train_loader=DataLoader(dataset=train_data,\n",
    "                        batch_size=BATCH_SIZE,\n",
    "                        shuffle=True,\n",
    "                        drop_last=True\n",
    "                       )\n",
    "valid_loader=DataLoader(dataset=valid_data,batch_size=BATCH_SIZE,drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 2/5 模型 ============================\n",
    "if torch.cuda.is_available():\n",
    "    model=nn.DataParallel(model)\n",
    "    model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 3/5 损失函数 ============================\n",
    "criterion=nn.CrossEntropyLoss()\n",
    "# ============================ step 4/5 优化器 ============================\n",
    "optimizer=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器\n",
    "\n",
    "# optimizer2=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gs(output,label):\n",
    "    s=0.0\n",
    "    for i,n in enumerate(label):\n",
    "        s+=output[i][n]\n",
    "    return s/(i+1)\n",
    "def gw(weights):\n",
    "    s=0\n",
    "    for i in weights:\n",
    "        if(i>0):\n",
    "            s+=i\n",
    "    return s\n",
    "def cw(grad_block,batch):\n",
    "    count_weight_list=[]\n",
    "    i=0\n",
    "    while(i<batch):\n",
    "        grads_val=grad_block[0][i].cpu().data.numpy().squeeze()\n",
    "        grads=grads_val.reshape(grads_val.shape[0],-1)\n",
    "        weights=np.mean(grads,axis=1)\n",
    "        w=gw(weights)\n",
    "        count_weight_list.append(w)\n",
    "        grads=0\n",
    "        grads_val=0\n",
    "        i+=1\n",
    "    return np.mean(count_weight_list)\n",
    "def count_final(grad_block,grad_block2,grad_block3,batch_size):\n",
    "    score_last=cw(grad_block,batch_size)\n",
    "    score_mid=cw(grad_block2,batch_size)\n",
    "    score_first=cw(grad_block3,batch_size)\n",
    "    w_last=score_last/(score_last+score_first+score_mid)\n",
    "    w_mid=score_mid/(score_last+score_first+score_mid)\n",
    "    w_first=score_first/(score_last+score_first+score_mid)\n",
    "    return w_first,w_mid,w_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:1,loss:1.2188\n",
      "0.4005208333333333\n",
      "epoch:1,loss:1.0881\n",
      "0.44765625\n",
      "============================================\n",
      "准确率由： 0.0 上升至： 0.453125 已更新并保存权值为weights/best.pkl\n",
      "第1个epoch的识别准确率为：45%\n",
      "epoch:2,loss:1.1982\n",
      "0.5223958333333333\n",
      "epoch:2,loss:1.1389\n",
      "0.51328125\n",
      "============================================\n",
      "准确率由： 0.453125 上升至： 0.5198317307692307 已更新并保存权值为weights/best.pkl\n",
      "第2个epoch的识别准确率为：51%\n",
      "epoch:3,loss:1.1810\n",
      "0.553125\n",
      "epoch:3,loss:1.0320\n",
      "0.5463541666666667\n",
      "============================================\n",
      "准确率由： 0.5198317307692307 上升至： 0.5406650641025641 已更新并保存权值为weights/best.pkl\n",
      "第3个epoch的识别准确率为：54%\n",
      "epoch:4,loss:1.0129\n",
      "0.5578125\n",
      "epoch:4,loss:0.9625\n",
      "0.5638020833333334\n",
      "============================================\n",
      "准确率由： 0.5406650641025641 上升至： 0.5592948717948718 已更新并保存权值为weights/best.pkl\n",
      "第4个epoch的识别准确率为：55%\n",
      "epoch:5,loss:1.2423\n",
      "0.5708333333333333\n",
      "epoch:5,loss:1.0992\n",
      "0.57578125\n",
      "============================================\n",
      "准确率由： 0.5592948717948718 上升至： 0.5725160256410257 已更新并保存权值为weights/best.pkl\n",
      "第5个epoch的识别准确率为：57%\n",
      "epoch:6,loss:1.0010\n",
      "0.6067708333333334\n",
      "epoch:6,loss:0.9287\n",
      "0.6057291666666667\n",
      "============================================\n",
      "准确率由： 0.5725160256410257 上升至： 0.5971554487179487 已更新并保存权值为weights/best.pkl\n",
      "第6个epoch的识别准确率为：59%\n",
      "epoch:7,loss:0.9492\n",
      "0.6072916666666667\n",
      "epoch:7,loss:1.0042\n",
      "0.6161458333333333\n",
      "============================================\n",
      "准确率由： 0.5971554487179487 上升至： 0.6137820512820513 已更新并保存权值为weights/best.pkl\n",
      "第7个epoch的识别准确率为：61%\n",
      "epoch:8,loss:0.9223\n",
      "0.6505208333333333\n",
      "epoch:8,loss:1.1376\n",
      "0.6390625\n",
      "============================================\n",
      "准确率由： 0.6137820512820513 上升至： 0.6386217948717948 已更新并保存权值为weights/best.pkl\n",
      "第8个epoch的识别准确率为：63%\n",
      "epoch:9,loss:1.0244\n",
      "0.6729166666666667\n",
      "epoch:9,loss:1.0109\n",
      "0.65546875\n",
      "============================================\n",
      "准确率由： 0.6386217948717948 上升至： 0.6488381410256411 已更新并保存权值为weights/best.pkl\n",
      "第9个epoch的识别准确率为：64%\n",
      "epoch:10,loss:0.8976\n",
      "0.690625\n",
      "epoch:10,loss:0.7595\n",
      "0.6859375\n",
      "============================================\n",
      "准确率由： 0.6488381410256411 上升至： 0.6822916666666666 已更新并保存权值为weights/best.pkl\n",
      "第10个epoch的识别准确率为：68%\n",
      "epoch:11,loss:0.8240\n",
      "0.7072916666666667\n",
      "epoch:11,loss:0.8298\n",
      "0.6942708333333333\n",
      "============================================\n",
      "准确率由： 0.6822916666666666 上升至： 0.6917067307692307 已更新并保存权值为weights/best.pkl\n",
      "第11个epoch的识别准确率为：69%\n",
      "epoch:12,loss:0.4076\n",
      "0.7453125\n",
      "epoch:12,loss:0.8016\n",
      "0.7401041666666667\n",
      "============================================\n",
      "准确率由： 0.6917067307692307 上升至： 0.7339743589743589 已更新并保存权值为weights/best.pkl\n",
      "第12个epoch的识别准确率为：73%\n",
      "epoch:13,loss:0.5674\n",
      "0.7744791666666667\n",
      "epoch:13,loss:0.6768\n",
      "0.7606770833333333\n",
      "============================================\n",
      "准确率由： 0.7339743589743589 上升至： 0.758613782051282 已更新并保存权值为weights/best.pkl\n",
      "第13个epoch的识别准确率为：75%\n",
      "epoch:14,loss:0.4576\n",
      "0.8125\n",
      "epoch:14,loss:0.4456\n",
      "0.7966145833333333\n",
      "============================================\n",
      "准确率由： 0.758613782051282 上升至： 0.796875 已更新并保存权值为weights/best.pkl\n",
      "第14个epoch的识别准确率为：79%\n",
      "epoch:15,loss:0.6802\n",
      "0.825\n",
      "epoch:15,loss:0.4659\n",
      "0.8174479166666667\n",
      "============================================\n",
      "准确率由： 0.796875 上升至： 0.8183092948717948 已更新并保存权值为weights/best.pkl\n",
      "第15个epoch的识别准确率为：81%\n",
      "epoch:16,loss:0.4073\n",
      "0.8625\n",
      "epoch:16,loss:0.3346\n",
      "0.83515625\n",
      "============================================\n",
      "准确率由： 0.8183092948717948 上升至： 0.8363381410256411 已更新并保存权值为weights/best.pkl\n",
      "第16个epoch的识别准确率为：83%\n",
      "epoch:17,loss:0.3950\n",
      "0.884375\n",
      "epoch:17,loss:0.5115\n",
      "0.8596354166666667\n",
      "============================================\n",
      "准确率由： 0.8363381410256411 上升至： 0.8573717948717948 已更新并保存权值为weights/best.pkl\n",
      "第17个epoch的识别准确率为：85%\n",
      "epoch:18,loss:0.4235\n",
      "0.8848958333333333\n",
      "epoch:18,loss:0.5130\n",
      "0.8833333333333333\n",
      "============================================\n",
      "准确率由： 0.8573717948717948 上升至： 0.8788060897435898 已更新并保存权值为weights/best.pkl\n",
      "第18个epoch的识别准确率为：87%\n",
      "epoch:19,loss:0.1735\n",
      "0.9046875\n",
      "epoch:19,loss:0.5203\n",
      "0.8911458333333333\n",
      "============================================\n",
      "准确率由： 0.8788060897435898 上升至： 0.8838141025641025 已更新并保存权值为weights/best.pkl\n",
      "第19个epoch的识别准确率为：88%\n",
      "epoch:20,loss:0.4146\n",
      "0.9057291666666667\n",
      "epoch:20,loss:0.2856\n",
      "0.8979166666666667\n",
      "============================================\n",
      "准确率由： 0.8838141025641025 上升至： 0.8916266025641025 已更新并保存权值为weights/best.pkl\n",
      "第20个epoch的识别准确率为：89%\n",
      "epoch:21,loss:0.3182\n",
      "0.9119791666666667\n",
      "epoch:21,loss:0.3413\n",
      "0.9158854166666667\n",
      "============================================\n",
      "准确率由： 0.8916266025641025 上升至： 0.9042467948717948 已更新并保存权值为weights/best.pkl\n",
      "第21个epoch的识别准确率为：90%\n",
      "epoch:22,loss:0.2552\n",
      "0.9052083333333333\n",
      "epoch:22,loss:0.1163\n",
      "0.9138020833333333\n",
      "============================================\n",
      "准确率由： 0.9042467948717948 上升至： 0.9120592948717948 已更新并保存权值为weights/best.pkl\n",
      "第22个epoch的识别准确率为：91%\n",
      "epoch:23,loss:0.0663\n",
      "0.9557291666666666\n",
      "epoch:23,loss:0.2692\n",
      "0.9388020833333334\n",
      "============================================\n",
      "准确率由： 0.9120592948717948 上升至： 0.9330929487179487 已更新并保存权值为weights/best.pkl\n",
      "第23个epoch的识别准确率为：93%\n",
      "epoch:24,loss:0.3037\n",
      "0.9364583333333333\n",
      "epoch:24,loss:0.2003\n",
      "0.9325520833333333\n",
      "============================================\n",
      "第24个epoch的识别准确率为：92%\n",
      "epoch:25,loss:0.1682\n",
      "0.9333333333333333\n",
      "epoch:25,loss:0.1067\n",
      "0.9333333333333333\n",
      "============================================\n",
      "第25个epoch的识别准确率为：92%\n",
      "epoch:26,loss:0.1506\n",
      "0.9390625\n",
      "epoch:26,loss:0.1805\n",
      "0.9411458333333333\n",
      "============================================\n",
      "准确率由： 0.9330929487179487 上升至： 0.9381009615384616 已更新并保存权值为weights/best.pkl\n",
      "第26个epoch的识别准确率为：93%\n",
      "epoch:27,loss:0.1558\n",
      "0.934375\n",
      "epoch:27,loss:0.0816\n",
      "0.9453125\n",
      "============================================\n",
      "准确率由： 0.9381009615384616 上升至： 0.9421073717948718 已更新并保存权值为weights/best.pkl\n",
      "第27个epoch的识别准确率为：94%\n",
      "epoch:28,loss:0.1645\n",
      "0.9510416666666667\n",
      "epoch:28,loss:0.2694\n",
      "0.94296875\n",
      "============================================\n",
      "第28个epoch的识别准确率为：93%\n",
      "epoch:29,loss:0.1621\n",
      "0.9395833333333333\n",
      "epoch:29,loss:0.1172\n",
      "0.93828125\n",
      "============================================\n",
      "第29个epoch的识别准确率为：93%\n",
      "epoch:30,loss:0.0692\n",
      "0.953125\n",
      "epoch:30,loss:0.2633\n",
      "0.9471354166666667\n",
      "============================================\n",
      "准确率由： 0.9421073717948718 上升至： 0.9497195512820513 已更新并保存权值为weights/best.pkl\n",
      "第30个epoch的识别准确率为：94%\n",
      "epoch:31,loss:0.0542\n",
      "0.9651041666666667\n",
      "epoch:31,loss:0.3592\n",
      "0.9541666666666667\n",
      "============================================\n",
      "准确率由： 0.9497195512820513 上升至： 0.9505208333333334 已更新并保存权值为weights/best.pkl\n",
      "第31个epoch的识别准确率为：95%\n",
      "epoch:32,loss:0.0453\n",
      "0.9609375\n",
      "epoch:32,loss:0.1363\n",
      "0.9520833333333333\n",
      "============================================\n",
      "第32个epoch的识别准确率为：94%\n",
      "epoch:33,loss:0.0546\n",
      "0.9625\n",
      "epoch:33,loss:0.1333\n",
      "0.9544270833333334\n",
      "============================================\n",
      "准确率由： 0.9505208333333334 上升至： 0.9513221153846154 已更新并保存权值为weights/best.pkl\n",
      "第33个epoch的识别准确率为：95%\n",
      "epoch:34,loss:0.3096\n",
      "0.959375\n",
      "epoch:34,loss:0.0776\n",
      "0.9591145833333333\n",
      "============================================\n",
      "准确率由： 0.9513221153846154 上升至： 0.9559294871794872 已更新并保存权值为weights/best.pkl\n",
      "第34个epoch的识别准确率为：95%\n",
      "epoch:35,loss:0.1811\n",
      "0.9625\n",
      "epoch:35,loss:0.1559\n",
      "0.9518229166666666\n",
      "============================================\n",
      "第35个epoch的识别准确率为：94%\n",
      "epoch:36,loss:0.2079\n",
      "0.9557291666666666\n",
      "epoch:36,loss:0.0898\n",
      "0.9606770833333333\n",
      "============================================\n",
      "准确率由： 0.9559294871794872 上升至： 0.9609375 已更新并保存权值为weights/best.pkl\n",
      "第36个epoch的识别准确率为：96%\n",
      "epoch:37,loss:0.1436\n",
      "0.9614583333333333\n",
      "epoch:37,loss:0.2053\n",
      "0.9497395833333333\n",
      "============================================\n",
      "第37个epoch的识别准确率为：95%\n",
      "epoch:38,loss:0.1169\n",
      "0.9609375\n",
      "epoch:38,loss:0.1579\n",
      "0.9572916666666667\n",
      "============================================\n",
      "第38个epoch的识别准确率为：95%\n",
      "epoch:39,loss:0.1055\n",
      "0.971875\n",
      "epoch:39,loss:0.0800\n",
      "0.96796875\n",
      "============================================\n",
      "准确率由： 0.9609375 上升至： 0.9633413461538461 已更新并保存权值为weights/best.pkl\n",
      "第39个epoch的识别准确率为：96%\n",
      "epoch:40,loss:0.0966\n",
      "0.9619791666666667\n",
      "epoch:40,loss:0.0416\n",
      "0.965625\n",
      "============================================\n",
      "准确率由： 0.9633413461538461 上升至： 0.9659455128205128 已更新并保存权值为weights/best.pkl\n",
      "第40个epoch的识别准确率为：96%\n",
      "epoch:41,loss:0.2914\n",
      "0.9619791666666667\n",
      "epoch:41,loss:0.1554\n",
      "0.9572916666666667\n",
      "============================================\n",
      "第41个epoch的识别准确率为：95%\n",
      "epoch:42,loss:0.2654\n",
      "0.9651041666666667\n",
      "epoch:42,loss:0.0552\n",
      "0.959375\n",
      "============================================\n",
      "第42个epoch的识别准确率为：95%\n",
      "epoch:43,loss:0.0136\n",
      "0.9734375\n",
      "epoch:43,loss:0.2287\n",
      "0.9661458333333334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================\n",
      "第43个epoch的识别准确率为：96%\n",
      "epoch:44,loss:0.1102\n",
      "0.9666666666666667\n",
      "epoch:44,loss:0.1641\n",
      "0.9604166666666667\n",
      "============================================\n",
      "第44个epoch的识别准确率为：95%\n",
      "epoch:45,loss:0.0363\n",
      "0.9760416666666667\n",
      "epoch:45,loss:0.2120\n",
      "0.9682291666666667\n",
      "============================================\n",
      "准确率由： 0.9659455128205128 上升至： 0.9689503205128205 已更新并保存权值为weights/best.pkl\n",
      "第45个epoch的识别准确率为：96%\n",
      "训练完毕，权重已保存为：weights/last.pkl\n"
     ]
    }
   ],
   "source": [
    "# ============================ step 5/5 训练 ============================\n",
    "# 记录每一次的数据，方便绘图\n",
    "train_curve=list()\n",
    "valid_curve=list()\n",
    "model.train()\n",
    "accurancy_global=0.0\n",
    "# model.layers[-1].register_forward_hook(farward_hook)\n",
    "# model.layers[-1].register_backward_hook(backward_hook)\n",
    "# model.layers[-5].register_backward_hook(backward2_hook)\n",
    "# model.layers[-11].register_backward_hook(backward3_hook)\n",
    "for epoch in range(MAX_EPOCH):\n",
    "    loss_mean=0.\n",
    "    correct=0.\n",
    "    total=0.\n",
    "    running_loss = 0.0\n",
    "\n",
    "    for i,data in enumerate(train_loader):\n",
    "        \n",
    "        img,label=data\n",
    "        img = Variable(img)\n",
    "        label = Variable(label)\n",
    "        if torch.cuda.is_available():\n",
    "            img=img.cuda()\n",
    "            label=label.cuda()\n",
    "        \n",
    "        out=model(img)\n",
    "        \n",
    "#         optimizer2.zero_grad()\n",
    "#         class_loss=gs(out,label)\n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        \n",
    "#         w_f,w_m,w_l=count_final(grad_block,grad_block2,grad_block3,BATCH_SIZE)\n",
    "        \n",
    "            \n",
    "#         grad_block=[]\n",
    "#         grad_block2=[]\n",
    "#         grad_block3=[]\n",
    "        \n",
    "        optimizer.zero_grad()  # 归0梯度\n",
    "        ###########################################\n",
    "#         print_loss=loss.data.item()\n",
    "        ###########################################\n",
    "        \n",
    "        loss=criterion(out,label)#得到损失函数\n",
    "        \n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        ###########################################\n",
    "        loss.backward()#反向传播\n",
    "        ###########################################\n",
    "        optimizer.step()#优化\n",
    "        \n",
    "        \n",
    "        if (i+1)%log_interval==0:\n",
    "            print('epoch:{},loss:{:.4f}'.format(epoch+1,loss.data.item()))\n",
    "        _, predicted = torch.max(out.data, 1)\n",
    "        total += label.size(0)\n",
    "#         if (i+1)%log_interval==0:\n",
    "#             print(\"============================================\")\n",
    "#             print(\"源数据标签：\",label)\n",
    "#             print(\"============================================\")\n",
    "#             print(\"预测结果：\",predicted)\n",
    "#             print(\"相等的结果为：\",predicted == label)\n",
    "        correct += (predicted == label).sum()\n",
    "        if (i+1)%log_interval==0:\n",
    "            print(correct.item() / total)\n",
    "#         print(correct.item())\n",
    "\n",
    "#     fi=epoch\n",
    "#     if (w_l>0.7)and(fi==0):\n",
    "#         fi+=1\n",
    "#         model.weights_try=w_f\n",
    "#         model.weights2_try=w_m\n",
    "#         model.weights3_try=w_l\n",
    "#         print(w_f,w_m,w_l)\n",
    "#         print('固定参数')\n",
    "#         print(fi)\n",
    "#     model.weights_try=w_f\n",
    "#     model.weights2_try=w_m\n",
    "#     model.weights3_try=w_l\n",
    "    print(\"============================================\")\n",
    "#     print(w_f,w_m,w_l)\n",
    "    accurancy=correct.item() / total\n",
    "    if accurancy>accurancy_global:\n",
    "        torch.save(model.state_dict(), './weights/best330.pkl')\n",
    "        print(\"准确率由：\", accurancy_global, \"上升至：\", accurancy, \"已更新并保存权值为weights/best.pkl\")\n",
    "        accurancy_global=accurancy\n",
    "    print('第%d个epoch的识别准确率为：%d%%' % (epoch + 1, 100*accurancy))\n",
    "# torch.save(model.state_dict(), './weights/last322.pkl')\n",
    "print(\"训练完毕，权重已保存为：weights/last.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.0'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
